{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook contains experiments for:\n",
    "\n",
    "* Loss functions\n",
    "* Learning rate decay\n",
    "* Weight initialization\n",
    "* Optimizers\n",
    "* Dropout"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# `lincoln` imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lincoln\n",
    "from lincoln.layers import Dense\n",
    "from lincoln.losses import SoftmaxCrossEntropy, MeanSquaredError\n",
    "from lincoln.optimizers import Optimizer, SGD, SGDMomentum\n",
    "from lincoln.activations import Sigmoid, Tanh, Linear, ReLU\n",
    "from lincoln.network import NeuralNetwork\n",
    "from lincoln.train import Trainer\n",
    "from lincoln.utils import mnist\n",
    "from lincoln.utils.np_utils import softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train, X_test, y_test = mnist.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60000"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_labels = len(y_train)\n",
    "num_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# one-hot encode\n",
    "num_labels = len(y_train)\n",
    "train_labels = np.zeros((num_labels, 10))\n",
    "for i in range(num_labels):\n",
    "    train_labels[i][y_train[i]] = 1\n",
    "\n",
    "num_labels = len(y_test)\n",
    "test_labels = np.zeros((num_labels, 10))\n",
    "for i in range(num_labels):\n",
    "    test_labels[i][y_test[i]] = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MNIST Demos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scale data to mean 0, variance 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test = X_train - np.mean(X_train), X_test - np.mean(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-33.318421449829934,\n",
       " 221.68157855017006,\n",
       " -33.318421449829934,\n",
       " 221.68157855017006)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.min(X_train), np.max(X_train), np.min(X_test), np.max(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test = X_train / np.std(X_train), X_test / np.std(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-0.424073894391566, 2.821543345689335, -0.424073894391566, 2.821543345689335)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.min(X_train), np.max(X_train), np.min(X_test), np.max(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_accuracy_model(model, test_set):\n",
    "    return print(f'''The model validation accuracy is: {np.equal(np.argmax(model.forward(test_set, inference=True), axis=1), y_test).sum() * 100.0 / test_set.shape[0]:.2f}%''')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Softmax cross entropy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trying sigmoid activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 0.611\n",
      "Validation loss after 20 epochs is 0.424\n",
      "Validation loss after 30 epochs is 0.388\n",
      "Validation loss after 40 epochs is 0.372\n",
      "Validation loss after 50 epochs is 0.364\n",
      "\n",
      "The model validation accuracy is: 72.80%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=89, \n",
    "                  activation=Tanh()),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Sigmoid())],\n",
    "            loss = MeanSquaredError(normalize=False), \n",
    "seed=20190119)\n",
    "\n",
    "trainer = Trainer(model, SGD(0.1))\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "            epochs = 50,\n",
    "            eval_every = 10,\n",
    "            seed=20190119,\n",
    "            batch_size=60);\n",
    "print()\n",
    "calc_accuracy_model(model, X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: even if we normalize the outputs of a classification model with mean squared error loss, it still doesn't help:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 0.952\n",
      "\n",
      "Loss increased after epoch 20, final loss was 0.952, \n",
      "using the model from epoch 10\n",
      "The model validation accuracy is: 41.73%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=89, \n",
    "                  activation=Tanh()),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Sigmoid())],\n",
    "            loss = MeanSquaredError(normalize=True), \n",
    "seed=20190119)\n",
    "\n",
    "trainer = Trainer(model, SGD(0.1))\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "            epochs = 50,\n",
    "            eval_every = 10,\n",
    "            seed=20190119,\n",
    "            batch_size=60);\n",
    "\n",
    "calc_accuracy_model(model, X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The reason is that we should be using softmax cross entropy loss!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trying sigmoid activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 1 epochs is 1.285\n",
      "Validation loss after 2 epochs is 0.970\n",
      "Validation loss after 3 epochs is 0.836\n",
      "Validation loss after 4 epochs is 0.763\n",
      "Validation loss after 5 epochs is 0.712\n",
      "Validation loss after 6 epochs is 0.679\n",
      "Validation loss after 7 epochs is 0.651\n",
      "Validation loss after 8 epochs is 0.631\n",
      "Validation loss after 9 epochs is 0.617\n",
      "Validation loss after 10 epochs is 0.599\n",
      "Validation loss after 11 epochs is 0.588\n",
      "Validation loss after 12 epochs is 0.576\n",
      "Validation loss after 13 epochs is 0.568\n",
      "Validation loss after 14 epochs is 0.557\n",
      "Validation loss after 15 epochs is 0.550\n",
      "Validation loss after 16 epochs is 0.544\n",
      "Validation loss after 17 epochs is 0.537\n",
      "Validation loss after 18 epochs is 0.533\n",
      "Validation loss after 19 epochs is 0.529\n",
      "Validation loss after 20 epochs is 0.523\n",
      "Validation loss after 21 epochs is 0.517\n",
      "Validation loss after 22 epochs is 0.512\n",
      "Validation loss after 23 epochs is 0.507\n",
      "\n",
      "Loss increased after epoch 24, final loss was 0.507, \n",
      "using the model from epoch 23\n",
      "\n",
      "The model validation accuracy is: 91.04%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=89, \n",
    "                  activation=Sigmoid()),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Linear())],\n",
    "            loss = SoftmaxCrossEntropy(), \n",
    "seed=20190119)\n",
    "\n",
    "trainer = Trainer(model, SGD(0.1))\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "            epochs = 130,\n",
    "            eval_every = 1,\n",
    "            seed=20190119,\n",
    "            batch_size=60);\n",
    "print()\n",
    "calc_accuracy_model(model, X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Trying ReLU activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 6.413\n",
      "\n",
      "Loss increased after epoch 20, final loss was 6.413, \n",
      "using the model from epoch 10\n",
      "\n",
      "The model validation accuracy is: 71.84%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=89, \n",
    "                  activation=ReLU()),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Linear())],\n",
    "            loss = SoftmaxCrossEntropy(), \n",
    "seed=20190119)\n",
    "\n",
    "trainer = Trainer(model, SGD(0.1))\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "            epochs = 50,\n",
    "            eval_every = 10,\n",
    "            seed=20190119,\n",
    "            batch_size=60);\n",
    "print()\n",
    "calc_accuracy_model(model, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 0.631\n",
      "Validation loss after 20 epochs is 0.580\n",
      "Validation loss after 30 epochs is 0.561\n",
      "Validation loss after 40 epochs is 0.560\n",
      "Validation loss after 50 epochs is 0.552\n",
      "\n",
      "The model validation accuracy is: 90.92%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=89, \n",
    "                  activation=Tanh()),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Linear())],\n",
    "            loss = SoftmaxCrossEntropy(), \n",
    "seed=20190119)\n",
    "\n",
    "trainer = Trainer(model, SGD(0.1))\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "            epochs = 50,\n",
    "            eval_every = 10,\n",
    "            seed=20190119,\n",
    "            batch_size=60);\n",
    "print()\n",
    "calc_accuracy_model(model, X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SGD Momentum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 1 epochs is 0.615\n",
      "Validation loss after 2 epochs is 0.489\n",
      "Validation loss after 3 epochs is 0.445\n",
      "\n",
      "Loss increased after epoch 4, final loss was 0.445, \n",
      "using the model from epoch 3\n",
      "The model validation accuracy is: 91.97%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=89, \n",
    "                  activation=Sigmoid()),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Linear())],\n",
    "            loss = SoftmaxCrossEntropy(), \n",
    "seed=20190119)\n",
    "\n",
    "optim = SGDMomentum(0.1, momentum=0.9)\n",
    "\n",
    "trainer = Trainer(model, SGDMomentum(0.1, momentum=0.9))\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "            epochs = 50,\n",
    "            eval_every = 1,\n",
    "            seed=20190119,\n",
    "            batch_size=60);\n",
    "\n",
    "calc_accuracy_model(model, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 0.387\n",
      "Validation loss after 20 epochs is 0.333\n",
      "Validation loss after 30 epochs is 0.316\n",
      "\n",
      "Loss increased after epoch 40, final loss was 0.316, \n",
      "using the model from epoch 30\n",
      "The model validation accuracy is: 95.36%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=89, \n",
    "                  activation=Tanh()),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Linear())],\n",
    "            loss = SoftmaxCrossEntropy(), \n",
    "seed=20190119)\n",
    "\n",
    "optim = SGD(0.1)\n",
    "\n",
    "optim = SGDMomentum(0.1, momentum=0.9)\n",
    "\n",
    "trainer = Trainer(model, SGDMomentum(0.1, momentum=0.9))\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "            epochs = 50,\n",
    "            eval_every = 10,\n",
    "            seed=20190119,\n",
    "            batch_size=60);\n",
    "\n",
    "calc_accuracy_model(model, X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Different weight decay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 0.389\n",
      "Validation loss after 20 epochs is 0.307\n",
      "Validation loss after 30 epochs is 0.290\n",
      "\n",
      "Loss increased after epoch 40, final loss was 0.290, \n",
      "using the model from epoch 30\n",
      "The model validation accuracy is: 95.98%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=89, \n",
    "                  activation=Tanh()),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Linear())],\n",
    "            loss = SoftmaxCrossEntropy(), \n",
    "seed=20190119)\n",
    "\n",
    "optimizer = SGDMomentum(0.15, momentum=0.9, final_lr = 0.05, decay_type='linear')\n",
    "\n",
    "trainer = Trainer(model, optimizer)\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "            epochs = 50,\n",
    "            eval_every = 10,\n",
    "            seed=20190119,\n",
    "            batch_size=60);\n",
    "\n",
    "calc_accuracy_model(model, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 0.450\n",
      "Validation loss after 20 epochs is 0.342\n",
      "Validation loss after 30 epochs is 0.296\n",
      "\n",
      "Loss increased after epoch 40, final loss was 0.296, \n",
      "using the model from epoch 30\n",
      "The model validation accuracy is: 95.75%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=89, \n",
    "                  activation=Tanh()),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Linear())],\n",
    "            loss = SoftmaxCrossEntropy(), \n",
    "seed=20190119)\n",
    "\n",
    "optimizer = SGDMomentum(0.2, \n",
    "                        momentum=0.9, \n",
    "                        final_lr = 0.05, \n",
    "                        decay_type='exponential')\n",
    "\n",
    "trainer = Trainer(model, optimizer)\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "            epochs = 50,\n",
    "            eval_every = 10,\n",
    "            seed=20190119,\n",
    "            batch_size=60);\n",
    "\n",
    "calc_accuracy_model(model, X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Changing weight init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 0.299\n",
      "Validation loss after 20 epochs is 0.226\n",
      "\n",
      "Loss increased after epoch 30, final loss was 0.226, \n",
      "using the model from epoch 20\n",
      "The model validation accuracy is: 96.69%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=89, \n",
    "                  activation=Tanh(),\n",
    "                  weight_init=\"glorot\"),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Linear(),\n",
    "                  weight_init=\"glorot\")],\n",
    "            loss = SoftmaxCrossEntropy(), \n",
    "seed=20190119)\n",
    "\n",
    "optimizer = SGDMomentum(0.15, momentum=0.9, final_lr = 0.05, decay_type='linear')\n",
    "\n",
    "trainer = Trainer(model, optimizer)\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "       epochs = 50,\n",
    "       eval_every = 10,\n",
    "       seed=20190119,\n",
    "           batch_size=60,\n",
    "           early_stopping=True);\n",
    "\n",
    "calc_accuracy_model(model, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 0.378\n",
      "Validation loss after 20 epochs is 0.264\n",
      "Validation loss after 30 epochs is 0.259\n",
      "\n",
      "Loss increased after epoch 40, final loss was 0.259, \n",
      "using the model from epoch 30\n",
      "The model validation accuracy is: 96.48%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=89, \n",
    "                  activation=Tanh(),\n",
    "                  weight_init=\"glorot\"),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Linear(),\n",
    "                  weight_init=\"glorot\")],\n",
    "            loss = SoftmaxCrossEntropy(), \n",
    "seed=20190119)\n",
    "\n",
    "trainer = Trainer(model, SGDMomentum(0.2, momentum=0.9, final_lr = 0.05, decay_type='exponential'))\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "       epochs = 50,\n",
    "       eval_every = 10,\n",
    "       seed=20190119,\n",
    "           batch_size=60,\n",
    "           early_stopping=True);\n",
    "\n",
    "calc_accuracy_model(model, X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 0.277\n",
      "Validation loss after 20 epochs is 0.233\n",
      "Validation loss after 30 epochs is 0.203\n",
      "Validation loss after 40 epochs is 0.201\n",
      "\n",
      "Loss increased after epoch 50, final loss was 0.201, \n",
      "using the model from epoch 40\n",
      "The model validation accuracy is: 96.77%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=89, \n",
    "                  activation=Tanh(),\n",
    "                  weight_init=\"glorot\",\n",
    "                  dropout=0.8),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Linear(),\n",
    "                  weight_init=\"glorot\")],\n",
    "            loss = SoftmaxCrossEntropy(), \n",
    "seed=20190119)\n",
    "\n",
    "trainer = Trainer(model, SGDMomentum(0.2, momentum=0.9, final_lr = 0.05, decay_type='exponential'))\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "       epochs = 50,\n",
    "       eval_every = 10,\n",
    "       seed=20190119,\n",
    "           batch_size=60,\n",
    "           early_stopping=True);\n",
    "\n",
    "calc_accuracy_model(model, X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deep Learning, with and without Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 0.315\n",
      "Validation loss after 20 epochs is 0.283\n",
      "Validation loss after 30 epochs is 0.258\n",
      "Validation loss after 40 epochs is 0.225\n",
      "Validation loss after 50 epochs is 0.204\n",
      "Validation loss after 60 epochs is 0.197\n",
      "Validation loss after 70 epochs is 0.175\n",
      "\n",
      "Loss increased after epoch 80, final loss was 0.175, \n",
      "using the model from epoch 70\n",
      "The model validation accuracy is: 97.34%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=178, \n",
    "                  activation=Tanh(),\n",
    "                  weight_init=\"glorot\",\n",
    "                  dropout=0.8),\n",
    "            Dense(neurons=46, \n",
    "                  activation=Tanh(),\n",
    "                  weight_init=\"glorot\",\n",
    "                  dropout=0.8),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Linear(),\n",
    "                  weight_init=\"glorot\")],\n",
    "            loss = SoftmaxCrossEntropy(), \n",
    "seed=20190119)\n",
    "\n",
    "trainer = Trainer(model, SGDMomentum(0.2, momentum=0.9, final_lr = 0.05, decay_type='exponential'))\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "       epochs = 100,\n",
    "       eval_every = 10,\n",
    "       seed=20190119,\n",
    "           batch_size=60,\n",
    "           early_stopping=True);\n",
    "\n",
    "calc_accuracy_model(model, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation loss after 10 epochs is 0.448\n",
      "Validation loss after 20 epochs is 0.348\n",
      "Validation loss after 30 epochs is 0.317\n",
      "Validation loss after 40 epochs is 0.302\n",
      "Validation loss after 50 epochs is 0.283\n",
      "Validation loss after 60 epochs is 0.249\n",
      "\n",
      "Loss increased after epoch 70, final loss was 0.249, \n",
      "using the model from epoch 60\n",
      "The model validation accuracy is: 96.08%\n"
     ]
    }
   ],
   "source": [
    "model = NeuralNetwork(\n",
    "    layers=[Dense(neurons=178, \n",
    "                  activation=Tanh(),\n",
    "                  weight_init=\"glorot\"),\n",
    "            Dense(neurons=46, \n",
    "                  activation=Tanh(),\n",
    "                  weight_init=\"glorot\"),\n",
    "            Dense(neurons=10, \n",
    "                  activation=Linear(),\n",
    "                  weight_init=\"glorot\")],\n",
    "            loss = SoftmaxCrossEntropy(), \n",
    "seed=20190119)\n",
    "\n",
    "trainer = Trainer(model, SGDMomentum(0.2, momentum=0.9, final_lr = 0.05, decay_type='exponential'))\n",
    "trainer.fit(X_train, train_labels, X_test, test_labels,\n",
    "       epochs = 100,\n",
    "       eval_every = 10,\n",
    "       seed=20190119,\n",
    "           batch_size=60,\n",
    "           early_stopping=True);\n",
    "\n",
    "calc_accuracy_model(model, X_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
